{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bbd712a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29275"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ord(\"牛\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "23a1bf74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'牛'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chr(29275)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "666a10ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\x00'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chr(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e8e227d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"'\\\\x00'\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chr(0).__repr__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4afe6021",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u0000\n"
     ]
    }
   ],
   "source": [
    "print(chr(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e0cadfff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this is a test\\x00string'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"this is a test\" + chr(0) + \"string\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8979aaf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a test\u0000string\n"
     ]
    }
   ],
   "source": [
    "print(\"this is a test\" + chr(0) + \"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c9f6445d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'hello! \\xe3\\x81\\x93\\xe3\\x82\\x93\\xe3\\x81\\xab\\xe3\\x81\\xa1\\xe3\\x81\\xaf!'\n"
     ]
    }
   ],
   "source": [
    "test_string = \"hello! こんにちは!\"\n",
    "utf8_encoded = test_string.encode(\"utf-8\")\n",
    "print(utf8_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e2716bf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bytes'>\n"
     ]
    }
   ],
   "source": [
    "print(type(utf8_encoded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "317675ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[104,\n",
       " 101,\n",
       " 108,\n",
       " 108,\n",
       " 111,\n",
       " 33,\n",
       " 32,\n",
       " 227,\n",
       " 129,\n",
       " 147,\n",
       " 227,\n",
       " 130,\n",
       " 147,\n",
       " 227,\n",
       " 129,\n",
       " 171,\n",
       " 227,\n",
       " 129,\n",
       " 161,\n",
       " 227,\n",
       " 129,\n",
       " 175,\n",
       " 33]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the byte values for the encoded string (integers from 0 to 255).\n",
    "list(utf8_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8f0299b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n",
      "23\n"
     ]
    }
   ],
   "source": [
    "print(len(test_string))\n",
    "print(len(utf8_encoded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c0d15aeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello! こんにちは!\n"
     ]
    }
   ],
   "source": [
    "print(utf8_encoded.decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e1319b39",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_utf8_bytes_to_str_wrong(bytestring: bytes):\n",
    "    return \"\".join([bytes([b]).decode(\"utf-8\") for b in bytestring])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "902ac8ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hello'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decode_utf8_bytes_to_str_wrong(\"hello\".encode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3a21c38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104\n",
      "101\n",
      "108\n",
      "108\n",
      "111\n",
      "33\n",
      "32\n",
      "227\n",
      "129\n",
      "147\n",
      "227\n",
      "130\n",
      "147\n",
      "227\n",
      "129\n",
      "171\n",
      "227\n",
      "129\n",
      "161\n",
      "227\n",
      "129\n",
      "175\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "for b in utf8_encoded:\n",
    "    print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ae02802",
   "metadata": {},
   "outputs": [
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf-8' codec can't decode byte 0xe3 in position 0: unexpected end of data",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[20], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m decode_utf8_bytes_to_str_wrong(utf8_encoded)\n",
      "Cell \u001b[1;32mIn[19], line 2\u001b[0m, in \u001b[0;36mdecode_utf8_bytes_to_str_wrong\u001b[1;34m(bytestring)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mdecode_utf8_bytes_to_str_wrong\u001b[39m(bytestring: \u001b[38;5;28mbytes\u001b[39m):\n\u001b[1;32m----> 2\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;28mbytes\u001b[39m([b])\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m b \u001b[38;5;129;01min\u001b[39;00m bytestring])\n",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0xe3 in position 0: unexpected end of data"
     ]
    }
   ],
   "source": [
    "decode_utf8_bytes_to_str_wrong(utf8_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73af686f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# utf-8 是可变长度的编码方式，不能逐个字节解码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "132ade46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ÀÀ'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = b\"\\xc0\\xc0\"\n",
    "a.decode(encoding=\"unicode_escape\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93261487",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc0 in position 0: invalid start byte\n",
      "Invalid UTF-8 sequence in hex: c0af\n"
     ]
    }
   ],
   "source": [
    "# 定义一个无效的两字节 UTF-8 序列\n",
    "invalid_utf8_sequence = b\"\\xc0\\xaf\"\n",
    "\n",
    "try:\n",
    "    # 尝试解码无效的 UTF-8 序列\n",
    "    decoded_string = invalid_utf8_sequence.decode(\"utf-8\")\n",
    "    print(f\"Decoded string: {decoded_string}\")\n",
    "except UnicodeDecodeError as e:\n",
    "    print(f\"UnicodeDecodeError: {e}\")\n",
    "\n",
    "# 打印无效的 UTF-8 序列的十六进制表示\n",
    "print(f\"Invalid UTF-8 sequence in hex: {invalid_utf8_sequence.hex()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f29f6df2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========\n",
      "re.compile('<start>|<end>|<pad>')\n",
      "String: This is a <start> sentence.\n",
      "Matches: ['<start>']\n",
      "\n",
      "String: Another <end> example.\n",
      "Matches: ['<end>']\n",
      "\n",
      "String: A <pad> token here.\n",
      "Matches: ['<pad>']\n",
      "\n",
      "String: No special tokens here.\n",
      "Matches: []\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# 示例特殊标记列表\n",
    "special_tokens = [\"<start>\", \"<end>\", \"<pad>\"]\n",
    "\n",
    "# 生成特殊标记的正则表达式模式\n",
    "special_pattern = (\n",
    "    re.compile(\"|\".join(re.escape(tok) for tok in special_tokens))\n",
    "    if special_tokens\n",
    "    else None\n",
    ")\n",
    "print(\"=\" * 10)\n",
    "print(special_pattern)\n",
    "# 测试正则表达式模式\n",
    "if special_pattern:\n",
    "    test_strings = [\n",
    "        \"This is a <start> sentence.\",\n",
    "        \"Another <end> example.\",\n",
    "        \"A <pad> token here.\",\n",
    "        \"No special tokens here.\",\n",
    "    ]\n",
    "\n",
    "    for s in test_strings:\n",
    "        matches = special_pattern.findall(s)\n",
    "        print(f\"String: {s}\")\n",
    "        print(f\"Matches: {matches}\\n\")\n",
    "else:\n",
    "    print(\"No special tokens provided.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54f08f43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subword: (b'H', b'e', b'l', b'l', b'o'), Frequency: 1\n",
      "Subword: (b' ',), Frequency: 2\n",
      "Subword: (b' ', b'w', b'o', b'r', b'l', b'd'), Frequency: 1\n",
      "Subword: (b'!',), Frequency: 1\n",
      "Subword: (b' ', b'T', b'h', b'i', b's'), Frequency: 1\n",
      "Subword: (b' ', b'i', b's'), Frequency: 1\n",
      "Subword: (b' ', b'a'), Frequency: 1\n",
      "Subword: (b' ', b't', b'e', b's', b't'), Frequency: 1\n",
      "Subword: (b'.',), Frequency: 1\n"
     ]
    }
   ],
   "source": [
    "import regex as re\n",
    "from typing import Dict, Tuple\n",
    "\n",
    "# 定义特殊标记的正则表达式模式\n",
    "special_tokens = [\"<start>\", \"<end>\", \"<pad>\"]\n",
    "special_pattern = re.compile(\"|\".join(re.escape(tok) for tok in special_tokens))\n",
    "\n",
    "# 定义用于进一步分词的正则表达式模式\n",
    "PAT = re.compile(\n",
    "    r\"'(?:[sdmt]|ll|ve|re)| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\n",
    ")\n",
    "\n",
    "\n",
    "def pre_tokenize_chunk(\n",
    "    chunk: str, special_pattern: re.Pattern | None\n",
    ") -> Dict[Tuple[bytes], int]:\n",
    "    \"\"\"Regex tokenizes the chunk. Splits first on special tokens, then uses PAT.\"\"\"\n",
    "    freqs: Dict[Tuple[bytes], int] = {}\n",
    "    sub_chunks = special_pattern.split(chunk) if special_pattern else [chunk]\n",
    "\n",
    "    for sub_chunk in sub_chunks:\n",
    "        for match in PAT.finditer(sub_chunk):\n",
    "            match_bytes = tuple(bytes([b]) for b in match.group().encode(\"UTF-8\"))\n",
    "            freqs[match_bytes] = freqs.get(match_bytes, 0) + 1\n",
    "\n",
    "    return freqs\n",
    "\n",
    "\n",
    "# 示例文本块\n",
    "chunk = \"Hello <start> world! This is a test <end>.\"\n",
    "\n",
    "# 调用 pre_tokenize_chunk 函数\n",
    "result = pre_tokenize_chunk(chunk, special_pattern)\n",
    "\n",
    "# 打印结果\n",
    "for k, v in result.items():\n",
    "    print(f\"Subword: {k}, Frequency: {v}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2e77eb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add(a: list, b: list):\n",
    "    a += b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76d31f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [1, 2]\n",
    "b = [3, 4]\n",
    "\n",
    "add(a, b)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91bcb341",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
